import scrapy

import logging
from copy import copy, deepcopy
import json
from HifoEsf.items import CaseItem

logger = logging.getLogger(__name__)


class AiwulongSpider(scrapy.Spider):
    name = 'aiwulong'
    allowed_domains = ['aiwulongrencai.com']
    start_urls = ['https://www.aiwulongrencai.com/plugin.php?id=fn_house&m=list&class=1']
    case_list_temp = 'https://www.aiwulongrencai.com/plugin.php?id=fn_house:Ajax&f=GetAjaxList&page={}&class=1&formhash={}'

    custom_settings = {
        'CONCURRENT_REQUESTS': 6,
        'DOWNLOADER_MIDDLEWARES': {
            'scrapy.downloadermiddlewares.retry.RetryMiddleware': None,
            'HifoEsf.middlewares.CustomRetryMiddleware': 500,
            'HifoEsf.middlewares.UserAgentMiddleware': 544,
            'HifoEsf.middlewares.OuterNetProxyMiddleware': 545,  # 启用redis中的代理，需要同时开启 RedisConnPipeline
            'scrapy_pyppeteer.downloadermiddlewares.PyppeteerMiddleware': 566,
        },
        'ITEM_PIPELINES': {
            'HifoEsf.pipelines.RedisConnPipeline': 299,  # 启用redis中的代理，需要同时开启 XXXProxyMiddleware
            'HifoEsf.pipelines.PcdPipeline': 300,  # 用于省市区id处理
            'HifoEsf.pipelines.AiwulongPipeline': 301,
            'HifoEsf.pipelines.MongoClientPipeline': 399,
        },
        'RETRY_TIMES': 20,
        'COOKIES_ENABLED': True,  # 携带cookies
    }

    def parse(self, response, **kwargs):
        # 提取页面隐藏参数
        param_formhash = response.xpath("//form//input[@name='formhash']/@value").extract_first()
        # 构造请求，获取案例列表首页
        page_num = 1
        headers = {
            'content-type': 'application/x-www-form-urlencoded; charset=UTF-8',
            'accept': 'application/json, text/javascript, */*; q=0.01',
            'accept-language': 'zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7',
        }
        yield scrapy.Request(
            self.case_list_temp.format(page_num - 1, param_formhash),
            headers=headers,
            callback=self.parse_case_list,
            meta=dict(page_num=deepcopy(page_num), param_formhash=param_formhash),
            dont_filter=True,
        )

    def parse_case_list(self, response):
        """
        获取案例列表
        :param response:
        :return:
        """
        page_num = copy(response.meta['page_num'])
        param_formhash = copy(response.meta['param_formhash'])
        try:
            resp_str = response.body.decode()
            case_list = json.loads(resp_str)
        except Exception as e:
            logging.error('第{}页 案例列表提取出错，error:{}'.format(page_num, e))
        else:
            if case_list:
                for case_dict in case_list:
                    item_case = CaseItem()
                    item_case['provinceName'] = '重庆'
                    item_case['cityName'] = '重庆'
                    item_case['districtName'] = '武隆'
                    item_case['caseId'] = case_dict.get('id')  # 案例id
                    item_case['title'] = case_dict.get('title')  # 标题
                    item_case['name'] = case_dict.get('small_area')  # 楼盘名称
                    item_case['address'] = case_dict.get('community')  # 楼盘地址
                    item_case['shangQuan'] = case_dict.get('province_text')  # 商圈
                    item_case['location'] = self.get_location(case_dict)  # 经纬度
                    item_case['totalPrice'] = case_dict.get('price')  # 总价
                    item_case['buildArea'] = case_dict.get('square')  # 建筑面积
                    item_case['floorLevels'] = case_dict.get('floor')  # 所在楼层数
                    item_case['totalFloor'] = case_dict.get('count_floor')  # 总楼层
                    item_case['bedroom'] = case_dict.get('room')  # 室
                    item_case['livingroom'] = case_dict.get('office')  # 厅
                    item_case['bathroom'] = case_dict.get('guard')  # 卫
                    item_case['roomType'] = case_dict.get('huxing')  # 户型
                    item_case['calQuality'] = case_dict.get('house_type')  # 住宅类别
                    item_case['decorationType'] = case_dict.get('decoration_type')  # 装修
                    item_case['directionType'] = self.get_orientation(case_dict.get('orientation'))  # 朝向
                    item_case['tag'] = case_dict.get('param', dict()).get('tag_list')  # 标签
                    item_case['dataUrl'] = case_dict.get('url')  # 案例url
                    item_case['caseType'] = self.get_case_type(case_dict.get('deal'))  # 案例类型
                    item_case['createDate'] = case_dict.get('dateline')  # 创建时间
                    item_case['listingDate'] = case_dict.get('updateline')  # 挂牌日期
                    yield item_case

                # 翻页
                next_page_num = page_num + 1
                headers = {
                    'content-type': 'application/x-www-form-urlencoded; charset=UTF-8',
                    'accept': 'application/json, text/javascript, */*; q=0.01',
                    'accept-language': 'zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7',
                }
                yield scrapy.Request(
                    self.case_list_temp.format(page_num, param_formhash),
                    headers=headers,
                    callback=self.parse_case_list,
                    meta=dict(page_num=deepcopy(next_page_num), param_formhash=param_formhash),
                    dont_filter=True,
                )

    @staticmethod
    def get_location(case_dict):
        """
        提取经纬度
        :param case_dict:
        :return:
        """
        try:
            lng = float(case_dict.get('lng'))
            lat = float(case_dict.get('lat'))
            assert lng and lat
        except:
            return
        else:
            return lng, lat

    @staticmethod
    def get_orientation(orientation):
        """
        提取朝向
        :param orientation:
        :return:
        """
        orientation_dict = {
            '0': None,
            '1': '朝东',
            '2': '朝西',
            '3': '朝南',
            '4': '朝北',
            '5': '东南',
            '6': '西南',
            '7': '东北',
            '8': '西北',
            '9': '南北',
        }
        try:
            orientation_ret = orientation_dict.get(orientation)
            assert orientation_ret
        except:
            return
        else:
            return orientation_ret

    @staticmethod
    def get_case_type(deal):
        """
        提取案例类型
        :param deal:
        :return:
        """
        try:
            deal = int(deal)
            ret = '成交' if deal else '挂牌'
        except:
            return '挂牌'
        else:
            return ret
